# -*- coding: utf-8 -*-
import json

import scrapy

from beike.item.village import VillageItem, VillageItemLoader
from database import sqlUtils
from domain.house import VillageBasic


class VillageSpider(scrapy.Spider):
    name = 'villageSpider'
    allowed_domains = ['ke.com']
    start_urls = ['https://gz.ke.com/xiaoqu/pg1']

    def parse(self, response):
        session = sqlUtils.get_mysql_sqlalchemy_session()
        # 抓取总页码
        page_div = response.css('.house-lst-page-box')
        dump = json.loads(page_div.attrib['page-data'])
        total_page_num = dump['totalPage']
        current_page_num = dump['curPage']

        # 获取当前小区个数
        item_nodes = response.css('.listContent li')
        for item_node in item_nodes:
            # 根据item文件中所定义的字段内容 ，进行字段内容的抓取
            item_loader = VillageItemLoader(item=VillageItem(), selector=item_node)
            item_loader.add_css("name", ".title a::text")
            item_loader.add_css("district", ".positionInfo .district::text")
            item_loader.add_css("bizcircle", ".positionInfo .bizcircle::text")
            item_loader.add_css("avgPrice", ".totalPrice span::text")
            item_loader.add_css("onSale", ".totalSellCount span::text")
            village_item = item_loader.load_item()
            # 获取页面id
            village_item['id'] = item_node.attrib['data-id']
            village = VillageBasic.tranfer(village_item)
            session.merge(village)
        if current_page_num != total_page_num:
            next_url = response.url.replace("pg{0}".format(current_page_num), "pg{0}".format(current_page_num + 1))
            yield scrapy.Request(url=next_url, callback=self.parse)
        session.commit()
        session.close()
        pass
